﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Xml;

namespace ReutersToHTML
{
    public class ReutersToHtmlConverter
    {
        Dictionary<string, IList<int>> _topics;
        private Dictionary<string, string> _fileCategories;

        [STAThread]
        public static void Main()
        {
            var rts = new ReutersToHtmlConverter
                          {
                              OutputFolder = @"C:\ParsingTest\reuters21578_HTML\"
                          };
            rts.Execute(Directory.GetFiles(@"C:\ParsingTest\reuters21578", "*.sgm"));

        }

        protected string OutputFolder { get; set; }

        public void Execute(IEnumerable<string> files)
        {
            _topics = new Dictionary<string, IList<int>>();
            _fileCategories = new Dictionary<string, string>();
            if (!Directory.Exists(OutputFolder))
            {
                Directory.CreateDirectory(OutputFolder);
            }
            int[] count = { files.Count() };
            ThreadPool.SetMaxThreads(count[0] * 2, 10);
            var mre = new AutoResetEvent(false);
            foreach (string file in files)
            {
                string myFile = file;
                ThreadPool.QueueUserWorkItem(delegate
                                                 {
                                                     ParseSgmFlile(myFile, true);
                                                     count[0]--;
                                                     mre.Set();

                                                 });
            }
            while (count[0] > 0)
            {
                mre.WaitOne();


            }
            WriteCategoryFiles();
        }

        private void WriteCategoryFiles()
        {
            var cat2FileMap = new StreamWriter(Path.Combine(OutputFolder, "Cat2File.txt"));
            foreach (KeyValuePair<string, IList<int>> pair in _topics)
            {
                cat2FileMap.Write(pair.Key);
                cat2FileMap.Write(' ');
                cat2FileMap.Write(pair.Value.Count);
                cat2FileMap.Write(' ');
                foreach (int value in pair.Value)
                {
                    cat2FileMap.Write(value);
                    cat2FileMap.Write(';');
                }
                cat2FileMap.WriteLine();
            }
            cat2FileMap.Close();

            var file2CatMap = new StreamWriter(Path.Combine(OutputFolder, "CategoryFile.txt"));
            foreach (KeyValuePair<string, string> pair in _fileCategories)
            {
                file2CatMap.WriteLine("{0} {1}", pair.Key, pair.Value);
            }
            file2CatMap.Close();
        }

        private void ParseSgmFlile(string fileName)
        {
            ParseSgmFlile(fileName, false);
        }

        private void ParseSgmFlile(string fileName, bool splitToFolders)
        {
            var sr = new StreamReader(fileName);
            //skip docType line
            sr.ReadLine();
            string xmlDoc = sr.ReadToEnd();
            xmlDoc = "<Collection>" + xmlDoc + "</Collection>";
            var doc = new XmlDocument();
            doc.Load(new StringReader(xmlDoc));
            var mre = new AutoResetEvent(false);
            if (doc.DocumentElement == null) throw new Exception(fileName +" is not a valid reuters document!");
            int[] count = { doc.DocumentElement.ChildNodes.Count };
            foreach (XmlNode reutersNode in doc.DocumentElement.ChildNodes)
            {
                XmlNode node = reutersNode;
                ThreadPool.QueueUserWorkItem(delegate
                                                 {
                                                     ReuetersDocument reuetersDocument = ReuetersDocument.GenerateDoc(node);
                                                     if (reuetersDocument != null)
                                                     {
                                                         AddToMap(reuetersDocument);

                                                         string htmlFile = ConvertToHtml(reuetersDocument);
                                                         if (splitToFolders)
                                                         {
                                                             MoveFileToCateogryFolder(htmlFile,
                                                                                      reuetersDocument.Topics.
                                                                                          FirstOrDefault());
                                                         }

                                                     }
                                                     lock (mre)
                                                     {
                                                         count[0]--;
                                                         mre.Set();
                                                     }
                                                 });
            }

            while (count[0] > 0)
            {
                mre.WaitOne();
            }
        }

        private void MoveFileToCateogryFolder(string file, string category)
        {
            if (string.IsNullOrEmpty(category)) category = "_none";
            var filename = Path.GetFileName(file);
            var targetFolder = Path.Combine(OutputFolder, category);
            if (!Directory.Exists(targetFolder))
            {
                Directory.CreateDirectory(targetFolder);
            }
            File.Copy(file, Path.Combine(targetFolder, filename), true);
            File.Delete(file);
        }

        private string ConvertToHtml(ReuetersDocument document)
        {
            string fileName = Path.Combine(OutputFolder, document.Name);
            var sw = new StreamWriter(fileName);
            sw.WriteLine("<HTML>");
            sw.WriteLine("<head>");
            sw.WriteLine("<title>");
            sw.WriteLine(document.Title);
            sw.WriteLine("</title>");
            sw.WriteLine("</head>");
            sw.WriteLine("<body>");
            sw.WriteLine(document.Body.Replace("\n", "<br>"));
            sw.WriteLine("</body>");
            sw.WriteLine("</html>");
            sw.Close();
            return fileName;
        }

        private void AddToMap(ReuetersDocument reuetersDocument)
        {
            foreach (string topic in reuetersDocument.Topics)
            {
                IList<int> list;
                lock (_topics)
                {
                    if (!_topics.TryGetValue(topic, out list))
                    {
                        list = new List<int>();
                        _topics[topic] = list;
                    }
                    if (list == null)
                    {
                        list = new List<int>();

                    }

                    list.Add(reuetersDocument.Id);
                }
            }
            if (reuetersDocument.Topics.Count > 0)
            {
                _fileCategories[string.Format("{0}_html_parsed.xml", reuetersDocument.Id)] =
                    reuetersDocument.Topics.First();
            }
        }
    }
}
